SHL github project: uat_shl

  • training module: shl_tm

  • prediction module: shl_pm

  • simulation module: shl_sm

  • misc module: shl_mm

data feeds:

  • historical bidding price, per second, time series

  • live bidding price, per second, time series

parameter lookup table: python dictionary

  • parm_si (seasonality index per second)

  • parm_month (parameter like alpha, beta, gamma, etc. per month)

[1] Import useful reference packages

Initialization


In [1]:
# function to fetch Seasonality-Index
def shl_intra_fetch_si(ccyy_mm, time, shl_data_parm_si):
#     return shl_data_parm_si[(shl_data_parm_si['ccyy-mm'] == '2017-09') & (shl_data_parm_si['time'] == '11:29:00')]
    return shl_data_parm_si[(shl_data_parm_si['ccyy-mm'] == ccyy_mm) & (shl_data_parm_si['time'] == time)].iloc[0]['si']

In [2]:
# function to fetch Dynamic-Increment
def shl_intra_fetch_di(ccyy_mm, shl_data_parm_month):
    return shl_data_parm_month[shl_data_parm_month['ccyy-mm'] == ccyy_mm].iloc[0]['di']

In [3]:
def shl_intra_fetch_previous_n_sec_time_as_str(shl_data_time_field, n):
    return str((pd.to_datetime(shl_data_time_field, format='%H:%M:%S') - pd.Timedelta(seconds=n)).time())

def shl_intra_fetch_future_n_sec_time_as_str(shl_data_time_field, n):
    return str((pd.to_datetime(shl_data_time_field, format='%H:%M:%S') - pd.Timedelta(seconds=-n)).time())

In [51]:
def shl_initialize(in_ccyy_mm='2017-07'):
    print()
    print('+-----------------------------------------------+')
    print('| shl_initialize()                              |')
    print('+-----------------------------------------------+')
    print()
    import pandas as pd
    global shl_data_parm_si
    global shl_data_parm_month
    shl_data_parm_si = pd.read_csv('data/parm_si.csv') 
    shl_data_parm_month = pd.read_csv('data/parm_month.csv') 

    global global_parm_ccyy_mm 
    global_parm_ccyy_mm = in_ccyy_mm
    
    # create default global base price
    global global_parm_base_price
    global_parm_base_price = 10000000

    global global_parm_dynamic_increment
    global_parm_dynamic_increment = shl_intra_fetch_di(global_parm_ccyy_mm, shl_data_parm_month)

    global global_parm_alpha
    global_parm_alpha = shl_data_parm_month[shl_data_parm_month['ccyy-mm'] == global_parm_ccyy_mm].iloc[0]['alpha']
    global global_parm_beta
    global_parm_beta  = shl_data_parm_month[shl_data_parm_month['ccyy-mm'] == global_parm_ccyy_mm].iloc[0]['beta']
    global global_parm_gamma
    global_parm_gamma = shl_data_parm_month[shl_data_parm_month['ccyy-mm'] == global_parm_ccyy_mm].iloc[0]['gamma']
    global global_parm_sec57_weight
    global_parm_sec57_weight = shl_data_parm_month[shl_data_parm_month['ccyy-mm'] == global_parm_ccyy_mm].iloc[0]['sec57-weight']
    global global_parm_month_weight
    global_parm_month_weight = shl_data_parm_month[shl_data_parm_month['ccyy-mm'] == global_parm_ccyy_mm].iloc[0]['month-weight']
    global global_parm_short_weight
    global_parm_short_weight = shl_data_parm_month[shl_data_parm_month['ccyy-mm'] == global_parm_ccyy_mm].iloc[0]['short-weight']

    # create default average error between 46~50 seconds:
    global global_parm_short_weight_misc
    global_parm_short_weight_misc = 0

    print('global_parm_ccyy_mm           : %s' % global_parm_ccyy_mm)
    print('-------------------------------------------------')
    print('global_parm_alpha             : %0.15f' % global_parm_alpha) # used in forecasting
    print('global_parm_beta              : %0.15f' % global_parm_beta)  # used in forecasting
    print('global_parm_gamma             : %0.15f' % global_parm_gamma) # used in forecasting
    print('global_parm_sec57_weight      : %f' % global_parm_sec57_weight) # used in training a model
    print('global_parm_month_weight      : %f' % global_parm_month_weight) # used in training a model
    print('global_parm_short_weight      : %f' % global_parm_short_weight) # used in training a model
    print('global_parm_dynamic_increment : %d' % global_parm_dynamic_increment)
    print('-------------------------------------------------')

#     plt.figure(figsize=(6,3)) # plot seasonality index
#     plt.plot(shl_data_parm_si[(shl_data_parm_si['ccyy-mm'] == global_parm_ccyy_mm)]['si'])
    
    global shl_data_shl_pm
    shl_data_shl_pm = pd.DataFrame() # initialize dataframe of prediction results
    print()
    print('prediction results dataframe: shl_data_shl_pm')
    print(shl_data_shl_pm)

In [62]:
global_parm_ccyy_mm


Out[62]:
'2017-07'

In [52]:
shl_initialize()


+-----------------------------------------------+
| shl_initialize()                              |
+-----------------------------------------------+

global_parm_ccyy_mm           : 2017-07
-------------------------------------------------
global_parm_alpha             : 0.636279780099081
global_parm_beta              : 0.237518711616408
global_parm_gamma             : 0.223562510966253
global_parm_sec57_weight      : 0.500000
global_parm_month_weight      : 0.900000
global_parm_short_weight      : 0.125000
global_parm_dynamic_increment : 300
-------------------------------------------------

prediction results dataframe: shl_data_shl_pm
Empty DataFrame
Columns: []
Index: []

In [53]:
shl_data_shl_pm


Out[53]:

shl_predict_price(in_current_time, in_current_price, in_k_sec) # return k-seconrds Predicted Prices, in a list format

shl_predict_set_price(in_current_time, in_current_price, in_k_sec) # return k-second Predicted Price + Dynamic Increment, in a list format

call k times of shl_predict_price()

shl_data_shl_pm_itr = pd.DataFrame() # initialize prediction dataframe at 11:29:00


In [63]:
def shl_predict_price(in_current_time, in_current_price, in_k_seconds=1):
# 11:29:00~11:29:50

    global shl_data_shl_pm
    
    global global_parm_short_weight_misc
    global_parm_short_weight_misc = 0
    
    global global_parm_base_price


    print()
    print('+-----------------------------------------------+')
    print('| shl_predict_price()                           |')
    print('+-----------------------------------------------+')
    print()
    print('   current_ccyy_mm: %s' % global_parm_ccyy_mm) # str, format: ccyy-mm
    print('in_current_time   : %s' % in_current_time) # str, format: hh:mm:ss
    print('in_current_price  : %d' % in_current_price) # number, format: integer
    print('in_k_seconds      : %d' % in_k_seconds) # number, format: integer
    print('-------------------------------------------------')

    
    # capture & calculate 11:29:00 bid price - 1 as base price
    if in_current_time == '11:29:00':
        global_parm_base_price = in_current_price -1 
        print('*INFO* At time [ %s ] Set global_parm_base_price : %d ' % (in_current_time, global_parm_base_price)) # Debug
        
#     f_actual_datetime = global_parm_ccyy_mm + ' ' + in_current_time
#     print('*INFO* f_actual_datetime   : %s ' %  f_actual_datetime)

    # get Seasonality-Index, for current second
    f_actual_si = shl_intra_fetch_si(global_parm_ccyy_mm, in_current_time, shl_data_parm_si)
    print('*INFO* f_actual_si         : %0.10f ' %  f_actual_si) # Debug
    
    # get Seasonality-Index, for current second + 1
    f_1_step_time = shl_intra_fetch_future_n_sec_time_as_str(in_current_time, 1)
    f_1_step_si = shl_intra_fetch_si(global_parm_ccyy_mm, f_1_step_time, shl_data_parm_si)
    print('*INFO* f_1_step_si         : %0.10f ' %  f_1_step_si) # Debug
    
    # calculate price increment: f_actual_price4pm
    f_actual_price4pm = in_current_price -  global_parm_base_price
    print('*INFO* f_actual_price4pm   : %d ' % f_actual_price4pm) # Debug
    
    # calculate seasonality adjusted price increment: f_actual_price4pmsi
    f_actual_price4pmsi = f_actual_price4pm / f_actual_si
    print('*INFO* f_actual_price4pmsi : %0.10f ' % f_actual_price4pmsi) # Debug
    

    if in_current_time == '11:29:00':
#         shl_data_shl_pm_itr = pd.DataFrame() # initialize prediction dataframe at 11:29:00
        print('---- call prediction function shl_pm ---- %s' % in_current_time)
        f_1_step_pred_les_level = f_actual_price4pmsi # special handling for 11:29:00
        f_1_step_pred_les_trend = 0 # special handling for 11:29:00
        f_1_step_pred_les = f_1_step_pred_les_level + f_1_step_pred_les_trend
        f_1_step_pred_adj_misc = 0
        f_1_step_pred_price_inc = (f_1_step_pred_les + f_1_step_pred_adj_misc) * f_1_step_si
        f_1_step_pred_price = f_1_step_pred_price_inc + global_parm_base_price
        f_1_step_pred_price_rounded = round(f_1_step_pred_price/100, 0) * 100
        f_1_step_pred_set_price_rounded = f_1_step_pred_price_rounded + global_parm_dynamic_increment
        
    else:
        print('---- call prediction function shl_pm ---- %s' % in_current_time)
#         function to get average forecast error between 46~50 seconds: mean(f_current_step_error)
        if in_current_time == '11:29:50':
            global_parm_short_weight_misc = (shl_data_shl_pm.iloc[46:50]['f_current_step_error'].sum() \
                                             + f_current_step_error) / 5
            print('*INFO* global_parm_short_weight_misc : %f' % global_parm_short_weight_misc)
           
        
#         p_1_step_time = shl_intra_fetch_previous_n_sec_time_as_str(in_current_time, 1)
        
#         previous_pred_les_level = shl_data_shl_pm[(shl_data_shl_pm['ccyy-mm'] == global_parm_ccyy_mm) \
#                                             & (shl_data_shl_pm['time'] ==p_1_step_time)].iloc[0]['f_1_step_pred_les_level']
#         print('     previous_pred_les_level : %f' % previous_pred_les_level)
        
#         previous_pred_les_trend = shl_data_shl_pm[(shl_data_shl_pm['ccyy-mm'] == global_parm_ccyy_mm) \
#                                             & (shl_data_shl_pm['time'] ==p_1_step_time)].iloc[0]['f_1_step_pred_les_trend']
#         print('     previous_pred_les_trend : %f' % previous_pred_les_trend)
        
#         p_1_step_time = shl_intra_fetch_previous_n_sec_time_as_str(in_current_time, 1)
        
        previous_pred_les_level = shl_data_shl_pm[(shl_data_shl_pm['ccyy-mm'] == global_parm_ccyy_mm) \
                                            & (shl_data_shl_pm['time'] ==in_current_time)].iloc[0]['f_1_step_pred_les_level']
        print('     previous_pred_les_level : %f' % previous_pred_les_level)
        
        previous_pred_les_trend = shl_data_shl_pm[(shl_data_shl_pm['ccyy-mm'] == global_parm_ccyy_mm) \
                                            & (shl_data_shl_pm['time'] ==in_current_time)].iloc[0]['f_1_step_pred_les_trend']
        print('     previous_pred_les_trend : %f' % previous_pred_les_trend)

            
        f_1_step_pred_les_level = global_parm_alpha * f_actual_price4pmsi \
                                    + (1 - global_parm_alpha) * (previous_pred_les_level + previous_pred_les_trend)
        print('     f_1_step_pred_les_level  : %f' % f_1_step_pred_les_level)
        f_1_step_pred_les_trend = global_parm_beta * (f_1_step_pred_les_level - previous_pred_les_level) \
                                    + (1 - global_parm_beta) * previous_pred_les_trend
        print('     f_1_step_pred_les_trend  : %f' % f_1_step_pred_les_trend)
        
        f_1_step_pred_les = f_1_step_pred_les_level + f_1_step_pred_les_trend
        f_1_step_pred_adj_misc = global_parm_short_weight_misc * global_parm_short_weight * global_parm_gamma
        f_1_step_pred_price_inc = (f_1_step_pred_les + f_1_step_pred_adj_misc) * f_1_step_si
        f_1_step_pred_price = f_1_step_pred_price_inc + global_parm_base_price
        f_1_step_pred_price_rounded = round(f_1_step_pred_price/100, 0) * 100
        f_1_step_pred_set_price_rounded = f_1_step_pred_price_rounded + global_parm_dynamic_increment
   
        
    # write results to shl_pm dataframe
            
    shl_data_shl_pm_itr_dict = {
                         'ccyy-mm' : global_parm_ccyy_mm
                        ,'time' : f_1_step_time # in_current_time + 1 second
                        ,'bid' : in_current_price
                        ,'f_actual_si' : f_actual_si
                        ,'f_1_step_si' : f_1_step_si
                        ,'f_actual_price4pm' : f_actual_price4pm
                        ,'f_actual_price4pmsi' :  f_actual_price4pmsi
                        ,'f_1_step_pred_les_level' : f_1_step_pred_les_level
                        ,'f_1_step_pred_les_trend' : f_1_step_pred_les_trend
                        ,'f_1_step_pred_les' : f_1_step_pred_les
                        ,'f_1_step_pred_adj_misc' : f_1_step_pred_adj_misc
                        ,'f_1_step_pred_price_inc' : f_1_step_pred_price_inc
                        ,'f_1_step_pred_price' : f_1_step_pred_price
                        ,'f_1_step_pred_price_rounded' : f_1_step_pred_price_rounded
                        ,'f_1_step_pred_set_price_rounded' : f_1_step_pred_set_price_rounded
                        }
#     shl_data_shl_pm_itr =  shl_data_shl_pm_itr.append(shl_data_shl_pm_itr_dict, ignore_index=True)
    shl_data_shl_pm     =  shl_data_shl_pm.append(shl_data_shl_pm_itr_dict, ignore_index=True)
    return shl_data_shl_pm_itr_dict

In [56]:
shl_predict_price('11:29:00', 90400, 1)


+-----------------------------------------------+
| shl_predict_price()                           |
+-----------------------------------------------+

   current_ccyy_mm: 2017-07
in_current_time   : 11:29:00
in_current_price  : 90400
in_k_seconds      : 1
-------------------------------------------------
*INFO* At time [ 11:29:00 ] Set global_parm_base_price : 90399 
*INFO* f_actual_si         : 0.0023669570 
*INFO* f_1_step_si         : 0.0223882810 
*INFO* f_actual_price4pm   : 1 
*INFO* f_actual_price4pmsi : 422.4833826724 
---- call prediction function shl_pm ---- 11:29:00
Out[56]:
{'bid': 90400,
 'ccyy-mm': '2017-07',
 'f_1_step_pred_adj_misc': 0,
 'f_1_step_pred_les': 422.48338267235101,
 'f_1_step_pred_les_level': 422.48338267235101,
 'f_1_step_pred_les_trend': 0,
 'f_1_step_pred_price': 90408.458676689101,
 'f_1_step_pred_price_inc': 9.458676689099125,
 'f_1_step_pred_price_rounded': 90400.0,
 'f_1_step_pred_set_price_rounded': 90700.0,
 'f_1_step_si': 0.022388280999999999,
 'f_actual_price4pm': 1,
 'f_actual_price4pmsi': 422.48338267235101,
 'f_actual_si': 0.0023669570000000003,
 'time': '11:29:01'}

In [64]:
shl_data_shl_pm


Out[64]:
bid ccyy-mm f_1_step_pred_adj_misc f_1_step_pred_les f_1_step_pred_les_level f_1_step_pred_les_trend f_1_step_pred_price f_1_step_pred_price_inc f_1_step_pred_price_rounded f_1_step_pred_set_price_rounded f_1_step_si f_actual_price4pm f_actual_price4pmsi f_actual_si time
0 90400.0 2017-07 0.0 422.483383 422.483383 0.0 90408.458677 9.458677 90400.0 90700.0 0.022388 1.0 422.483383 0.002367 11:29:01

In [68]:
shl_predict_price('11:29:01', 90400, 3)


+-----------------------------------------------+
| shl_predict_price()                           |
+-----------------------------------------------+

   current_ccyy_mm: 2017-07
in_current_time   : 11:29:01
in_current_price  : 90400
in_k_seconds      : 3
-------------------------------------------------
*INFO* f_actual_si         : 0.0223882810 
*INFO* f_1_step_si         : 0.0309107700 
*INFO* f_actual_price4pm   : 1 
*INFO* f_actual_price4pmsi : 44.6662251559 
---- call prediction function shl_pm ---- 11:29:01
     previous_pred_les_level : 422.483383
     previous_pred_les_trend : 0.000000
     f_1_step_pred_les_level  : 182.085965
     f_1_step_pred_les_trend  : -57.098885
Out[68]:
{'bid': 90400,
 'ccyy-mm': '2017-07',
 'f_1_step_pred_adj_misc': 0.0,
 'f_1_step_pred_les': 124.98707979409099,
 'f_1_step_pred_les_level': 182.08596477013626,
 'f_1_step_pred_les_trend': -57.098884976045262,
 'f_1_step_pred_price': 90402.863446876494,
 'f_1_step_pred_price_inc': 3.863446876486794,
 'f_1_step_pred_price_rounded': 90400.0,
 'f_1_step_pred_set_price_rounded': 90700.0,
 'f_1_step_si': 0.030910770000000001,
 'f_actual_price4pm': 1,
 'f_actual_price4pmsi': 44.66622515591974,
 'f_actual_si': 0.022388280999999999,
 'time': '11:29:02'}

In [69]:
shl_data_shl_pm


Out[69]:
bid ccyy-mm f_1_step_pred_adj_misc f_1_step_pred_les f_1_step_pred_les_level f_1_step_pred_les_trend f_1_step_pred_price f_1_step_pred_price_inc f_1_step_pred_price_rounded f_1_step_pred_set_price_rounded f_1_step_si f_actual_price4pm f_actual_price4pmsi f_actual_si time
0 90400.0 2017-07 0.0 422.483383 422.483383 0.000000 90408.458677 9.458677 90400.0 90700.0 0.022388 1.0 422.483383 0.002367 11:29:01
1 90400.0 2017-07 0.0 124.987080 182.085965 -57.098885 90402.863447 3.863447 90400.0 90700.0 0.030911 1.0 44.666225 0.022388 11:29:02
2 90400.0 2017-07 0.0 124.987080 182.085965 -57.098885 90402.863447 3.863447 90400.0 90700.0 0.030911 1.0 44.666225 0.022388 11:29:02
3 90400.0 2017-07 0.0 124.987080 182.085965 -57.098885 90402.863447 3.863447 90400.0 90700.0 0.030911 1.0 44.666225 0.022388 11:29:02

In [76]:
shl_data_shl_pm = pd.DataFrame()
for i in range(0,10):
    print('\n11:29:0'+str(i), 90400, 1)
    shl_predict_price('11:29:0'+str(i), 90400, 1)


11:29:00 90400 1

+-----------------------------------------------+
| shl_predict_price()                           |
+-----------------------------------------------+

   current_ccyy_mm: 2017-07
in_current_time   : 11:29:00
in_current_price  : 90400
in_k_seconds      : 1
-------------------------------------------------
*INFO* At time [ 11:29:00 ] Set global_parm_base_price : 90399 
*INFO* f_actual_si         : 0.0023669570 
*INFO* f_1_step_si         : 0.0223882810 
*INFO* f_actual_price4pm   : 1 
*INFO* f_actual_price4pmsi : 422.4833826724 
---- call prediction function shl_pm ---- 11:29:00

11:29:01 90400 1

+-----------------------------------------------+
| shl_predict_price()                           |
+-----------------------------------------------+

   current_ccyy_mm: 2017-07
in_current_time   : 11:29:01
in_current_price  : 90400
in_k_seconds      : 1
-------------------------------------------------
*INFO* f_actual_si         : 0.0223882810 
*INFO* f_1_step_si         : 0.0309107700 
*INFO* f_actual_price4pm   : 1 
*INFO* f_actual_price4pmsi : 44.6662251559 
---- call prediction function shl_pm ---- 11:29:01
     previous_pred_les_level : 422.483383
     previous_pred_les_trend : 0.000000
     f_1_step_pred_les_level  : 182.085965
     f_1_step_pred_les_trend  : -57.098885

11:29:02 90400 1

+-----------------------------------------------+
| shl_predict_price()                           |
+-----------------------------------------------+

   current_ccyy_mm: 2017-07
in_current_time   : 11:29:02
in_current_price  : 90400
in_k_seconds      : 1
-------------------------------------------------
*INFO* f_actual_si         : 0.0309107700 
*INFO* f_1_step_si         : 0.0377696020 
*INFO* f_actual_price4pm   : 1 
*INFO* f_actual_price4pmsi : 32.3511837460 
---- call prediction function shl_pm ---- 11:29:02
     previous_pred_les_level : 182.085965
     previous_pred_les_trend : -57.098885
     f_1_step_pred_les_level  : 66.044732
     f_1_step_pred_les_trend  : -71.098795

11:29:03 90400 1

+-----------------------------------------------+
| shl_predict_price()                           |
+-----------------------------------------------+

   current_ccyy_mm: 2017-07
in_current_time   : 11:29:03
in_current_price  : 90400
in_k_seconds      : 1
-------------------------------------------------
*INFO* f_actual_si         : 0.0377696020 
*INFO* f_1_step_si         : 0.0457052300 
*INFO* f_actual_price4pm   : 1 
*INFO* f_actual_price4pmsi : 26.4763181778 
---- call prediction function shl_pm ---- 11:29:03
     previous_pred_les_level : 66.044732
     previous_pred_les_trend : -71.098795
     f_1_step_pred_les_level  : 15.008081
     f_1_step_pred_les_trend  : -66.333661

11:29:04 90400 1

+-----------------------------------------------+
| shl_predict_price()                           |
+-----------------------------------------------+

   current_ccyy_mm: 2017-07
in_current_time   : 11:29:04
in_current_price  : 90400
in_k_seconds      : 1
-------------------------------------------------
*INFO* f_actual_si         : 0.0457052300 
*INFO* f_1_step_si         : 0.0452799070 
*INFO* f_actual_price4pm   : 1 
*INFO* f_actual_price4pmsi : 21.8793341594 
---- call prediction function shl_pm ---- 11:29:04
     previous_pred_les_level : 15.008081
     previous_pred_les_trend : -66.333661
     f_1_step_pred_les_level  : -4.746773
     f_1_step_pred_les_trend  : -55.270323

11:29:05 90400 1

+-----------------------------------------------+
| shl_predict_price()                           |
+-----------------------------------------------+

   current_ccyy_mm: 2017-07
in_current_time   : 11:29:05
in_current_price  : 90400
in_k_seconds      : 1
-------------------------------------------------
*INFO* f_actual_si         : 0.0452799070 
*INFO* f_1_step_si         : 0.0807556680 
*INFO* f_actual_price4pm   : 1 
*INFO* f_actual_price4pmsi : 22.0848510135 
---- call prediction function shl_pm ---- 11:29:05
     previous_pred_les_level : -4.746773
     previous_pred_les_trend : -55.270323
     f_1_step_pred_les_level  : -7.777287
     f_1_step_pred_les_trend  : -42.862391

11:29:06 90400 1

+-----------------------------------------------+
| shl_predict_price()                           |
+-----------------------------------------------+

   current_ccyy_mm: 2017-07
in_current_time   : 11:29:06
in_current_price  : 90400
in_k_seconds      : 1
-------------------------------------------------
*INFO* f_actual_si         : 0.0807556680 
*INFO* f_1_step_si         : 0.0985017130 
*INFO* f_actual_price4pm   : 1 
*INFO* f_actual_price4pmsi : 12.3830317396 
---- call prediction function shl_pm ---- 11:29:06
     previous_pred_les_level : -7.777287
     previous_pred_les_trend : -42.862391
     f_1_step_pred_les_level  : -10.539602
     f_1_step_pred_les_trend  : -33.337872

11:29:07 90400 1

+-----------------------------------------------+
| shl_predict_price()                           |
+-----------------------------------------------+

   current_ccyy_mm: 2017-07
in_current_time   : 11:29:07
in_current_price  : 90400
in_k_seconds      : 1
-------------------------------------------------
*INFO* f_actual_si         : 0.0985017130 
*INFO* f_1_step_si         : 0.1361543100 
*INFO* f_actual_price4pm   : 1 
*INFO* f_actual_price4pmsi : 10.1521077100 
---- call prediction function shl_pm ---- 11:29:07
     previous_pred_les_level : -10.539602
     previous_pred_les_trend : -33.337872
     f_1_step_pred_les_level  : -9.499544
     f_1_step_pred_les_trend  : -25.172470

11:29:08 90400 1

+-----------------------------------------------+
| shl_predict_price()                           |
+-----------------------------------------------+

   current_ccyy_mm: 2017-07
in_current_time   : 11:29:08
in_current_price  : 90400
in_k_seconds      : 1
-------------------------------------------------
*INFO* f_actual_si         : 0.1361543100 
*INFO* f_1_step_si         : 0.2041642360 
*INFO* f_actual_price4pm   : 1 
*INFO* f_actual_price4pmsi : 7.3446077469 
---- call prediction function shl_pm ---- 11:29:08
     previous_pred_les_level : -9.499544
     previous_pred_les_trend : -25.172470
     f_1_step_pred_les_level  : -7.937687
     f_1_step_pred_les_trend  : -18.822568

11:29:09 90400 1

+-----------------------------------------------+
| shl_predict_price()                           |
+-----------------------------------------------+

   current_ccyy_mm: 2017-07
in_current_time   : 11:29:09
in_current_price  : 90400
in_k_seconds      : 1
-------------------------------------------------
*INFO* f_actual_si         : 0.2041642360 
*INFO* f_1_step_si         : 0.2310771670 
*INFO* f_actual_price4pm   : 1 
*INFO* f_actual_price4pmsi : 4.8980174961 
---- call prediction function shl_pm ---- 11:29:09
     previous_pred_les_level : -7.937687
     previous_pred_les_trend : -18.822568
     f_1_step_pred_les_level  : -6.616736
     f_1_step_pred_les_trend  : -14.038105

In [ ]:


In [77]:
# shl_data_shl_pm = pd.DataFrame()

In [78]:
shl_data_shl_pm


Out[78]:
bid ccyy-mm f_1_step_pred_adj_misc f_1_step_pred_les f_1_step_pred_les_level f_1_step_pred_les_trend f_1_step_pred_price f_1_step_pred_price_inc f_1_step_pred_price_rounded f_1_step_pred_set_price_rounded f_1_step_si f_actual_price4pm f_actual_price4pmsi f_actual_si time
0 90400.0 2017-07 0.0 422.483383 422.483383 0.000000 90408.458677 9.458677 90400.0 90700.0 0.022388 1.0 422.483383 0.002367 11:29:01
1 90400.0 2017-07 0.0 124.987080 182.085965 -57.098885 90402.863447 3.863447 90400.0 90700.0 0.030911 1.0 44.666225 0.022388 11:29:02
2 90400.0 2017-07 0.0 -5.054063 66.044732 -71.098795 90398.809110 -0.190890 90400.0 90700.0 0.037770 1.0 32.351184 0.030911 11:29:03
3 90400.0 2017-07 0.0 -51.325580 15.008081 -66.333661 90396.654153 -2.345847 90400.0 90700.0 0.045705 1.0 26.476318 0.037770 11:29:04
4 90400.0 2017-07 0.0 -60.017096 -4.746773 -55.270323 90396.282431 -2.717569 90400.0 90700.0 0.045280 1.0 21.879334 0.045705 11:29:05
5 90400.0 2017-07 0.0 -50.639678 -7.777287 -42.862391 90394.910559 -4.089441 90400.0 90700.0 0.080756 1.0 22.084851 0.045280 11:29:06
6 90400.0 2017-07 0.0 -43.877474 -10.539602 -33.337872 90394.677994 -4.322006 90400.0 90700.0 0.098502 1.0 12.383032 0.080756 11:29:07
7 90400.0 2017-07 0.0 -34.672014 -9.499544 -25.172470 90394.279256 -4.720744 90400.0 90700.0 0.136154 1.0 10.152108 0.098502 11:29:08
8 90400.0 2017-07 0.0 -26.760255 -7.937687 -18.822568 90393.536513 -5.463487 90400.0 90700.0 0.204164 1.0 7.344608 0.136154 11:29:09
9 90400.0 2017-07 0.0 -20.654841 -6.616736 -14.038105 90394.227138 -4.772862 90400.0 90700.0 0.231077 1.0 4.898017 0.204164 11:29:10

In [ ]:


In [ ]:


In [ ]:


In [ ]:
%matplotlib inline
import matplotlib.pyplot as plt

In [19]:
import pandas as pd

Read raw data


In [ ]:
shl_data_history_ts_process = pd.read_csv('data/history_ts.csv') 
shl_data_history_ts_process.tail()

In [ ]:
shl_data_history_table_process = pd.read_csv('data/history_table.csv') 
shl_data_history_table_process.tail()

In [ ]:
shl_data_parm_si = pd.read_csv('data/parm_si.csv') 
# print(shl_data_parm_si[(shl_data_parm_si['ccyy-mm'] == '2017-07') & (shl_data_parm_si['time'] == '11:29:00')].iloc[0]['si'])
shl_data_parm_si.tail()

In [ ]:
shl_data_parm_month = pd.read_csv('data/parm_month.csv') 
# print(shl_data_parm_month[(shl_data_parm_month['ccyy-mm'] == '2017-07') & (shl_data_parm_month['time'] == '11:29:00')].iloc[0]['di'])
shl_data_parm_month.tail()

Initialization


In [ ]:
# function to fetch Seasonality-Index
def shl_intra_fetch_si(ccyy_mm, time, shl_data_parm_si):
#     return shl_data_parm_si[(shl_data_parm_si['ccyy-mm'] == '2017-09') & (shl_data_parm_si['time'] == '11:29:00')]
    return shl_data_parm_si[(shl_data_parm_si['ccyy-mm'] == ccyy_mm) & (shl_data_parm_si['time'] == time)].iloc[0]['si']

In [ ]:
# function to fetch Dynamic-Increment
def shl_intra_fetch_di(ccyy_mm, shl_data_parm_month):
#     print(shl_data_parm_month[shl_data_parm_month['ccyy-mm'] == '2017-07'].iloc[0]['di'])
    return shl_data_parm_month[shl_data_parm_month['ccyy-mm'] == ccyy_mm].iloc[0]['di']

In [ ]:
def shl_intra_fetch_previous_n_sec_time_as_str(shl_data_time_field, n):
    return str((pd.to_datetime(shl_data_time_field, format='%H:%M:%S') - pd.Timedelta(seconds=n)).time())
# print(shl_intra_fetch_previous_n_sec_time_as_str('11:29:57',3))

def shl_intra_fetch_future_n_sec_time_as_str(shl_data_time_field, n):
    return str((pd.to_datetime(shl_data_time_field, format='%H:%M:%S') - pd.Timedelta(seconds=-n)).time())
# print(shl_intra_fetch_future_n_sec_time_as_str('11:29:57',3))

shl_sm Simulation Module Parm:


In [ ]:
# which month to predict?
global_parm_ccyy_mm = '2017-04'
global_parm_ccyy_mm_offset = 1647

# global_parm_ccyy_mm = '2017-05'
# global_parm_ccyy_mm_offset = 1708

# global_parm_ccyy_mm = '2017-06'
# global_parm_ccyy_mm_offset = 1769

# global_parm_ccyy_mm = '2017-07'
# global_parm_ccyy_mm_offset = 1830

shl_pm Prediction Module Parm:


In [ ]:
# create default global base price
global_parm_base_price = 10000000

global_parm_dynamic_increment = shl_intra_fetch_di(global_parm_ccyy_mm, shl_data_parm_month)

global_parm_alpha = shl_data_parm_month[shl_data_parm_month['ccyy-mm'] == global_parm_ccyy_mm].iloc[0]['alpha']
global_parm_beta  = shl_data_parm_month[shl_data_parm_month['ccyy-mm'] == global_parm_ccyy_mm].iloc[0]['beta']
global_parm_gamma = shl_data_parm_month[shl_data_parm_month['ccyy-mm'] == global_parm_ccyy_mm].iloc[0]['gamma']
global_parm_sec57_weight = shl_data_parm_month[shl_data_parm_month['ccyy-mm'] == global_parm_ccyy_mm].iloc[0]['sec57-weight']
global_parm_month_weight = shl_data_parm_month[shl_data_parm_month['ccyy-mm'] == global_parm_ccyy_mm].iloc[0]['month-weight']
global_parm_short_weight = shl_data_parm_month[shl_data_parm_month['ccyy-mm'] == global_parm_ccyy_mm].iloc[0]['short-weight']

# create default average error between 46~50 seconds:
global_parm_short_weight_misc = 0

print('=================================================')
print('  Global Parameters for Month : %s' % global_parm_ccyy_mm)
print('-------------------------------------------------')

print('global_parm_dynamic_increment : %d' % global_parm_dynamic_increment)

print('global_parm_alpha             : %0.15f' % global_parm_alpha) # used in forecasting
print('global_parm_beta              : %0.15f' % global_parm_beta)  # used in forecasting
print('global_parm_gamma             : %0.15f' % global_parm_gamma) # used in forecasting
print('global_parm_sec57_weight      : %f' % global_parm_sec57_weight) # used in training a model
print('global_parm_month_weight      : %f' % global_parm_month_weight) # used in training a model
print('global_parm_short_weight      : %f' % global_parm_short_weight) # used in training a model
print('=================================================')

# plot seasonality index
plt.figure(figsize=(6,3))
plt.plot(shl_data_parm_si[(shl_data_parm_si['ccyy-mm'] == global_parm_ccyy_mm)]['si'])

In [ ]:

Start of shl_sm


In [ ]:
# 11:29:00~11:29:50

global_parm_short_weight_misc = 0

# for i in range(1830, 1830+51): # use July 2015 data as simulatino
for i in range(global_parm_ccyy_mm_offset, global_parm_ccyy_mm_offset+51): # use July 2015 data as simulatino
    print('\n<<<< Record No.: %5d >>>>' % i)
    print(shl_data_history_ts_process['ccyy-mm'][i]) # format: ccyy-mm
    print(shl_data_history_ts_process['time'][i]) # format: hh:mm:ss
    print(shl_data_history_ts_process['bid-price'][i]) # format: integer
#     print(shl_data_history_ts_process['ref-price'][i])
    
    # capture & calculate 11:29:00 bid price - 1 = base price
    if shl_data_history_ts_process['time'][i] == '11:29:00':
        global_parm_base_price = shl_data_history_ts_process['bid-price'][i] -1 
        print('*INFO* global_parm_base_price : %d ' % global_parm_base_price)

        
    print('---- Pre-Process ---')
    # pre-process: ccyy-mm-hh:mm:ss
    f_actual_datetime = shl_data_history_ts_process['ccyy-mm'][i] + ' ' + shl_data_history_ts_process['time'][i]
    f_actual_price4pm = shl_data_history_ts_process['bid-price'][i] -  global_parm_base_price
    print('*INFO* f_actual_datetime   : %s ' %  f_actual_datetime)
    print('*INFO* f_actual_price4pm   : %d ' % f_actual_price4pm)
    
    # get Seasonality-Index
    f_actual_si = shl_intra_fetch_si(shl_data_history_ts_process['ccyy-mm'][i]
                                         ,shl_data_history_ts_process['time'][i]
                                         ,shl_data_parm_si)
    print('*INFO* f_actual_si         : %0.10f ' %  f_actual_si)
    f_1_step_si = shl_intra_fetch_si(shl_data_history_ts_process['ccyy-mm'][i]
                                         ,shl_data_history_ts_process['time'][i+1]
                                         ,shl_data_parm_si)
    print('*INFO* f_1_step_si         : %0.10f ' %  f_1_step_si)
    # get de-seasoned price: price4pmsi
    f_actual_price4pmsi = f_actual_price4pm / f_actual_si
    print('*INFO* f_actual_price4pmsi : %0.10f ' % f_actual_price4pmsi)
    


    if shl_data_history_ts_process['time'][i] == '11:29:00':
        shl_data_shl_pm = pd.DataFrame() # initialize prediction dataframe at 11:29:00
        print('---- call prediction function shl_pm ---- %s' % shl_data_history_ts_process['time'][i])
        f_1_step_pred_les_level = f_actual_price4pmsi
        f_1_step_pred_les_trend = 0
        f_1_step_pred_les = f_1_step_pred_les_level + f_1_step_pred_les_trend
        f_1_step_pred_adj_misc = 0
#         f_1_step_pred_price_inc = (f_1_step_pred_les + f_1_step_pred_adj_misc) * f_actual_si
        f_1_step_pred_price_inc = (f_1_step_pred_les + f_1_step_pred_adj_misc) * f_1_step_si
        f_1_step_pred_price = f_1_step_pred_price_inc + global_parm_base_price
        f_1_step_pred_price_rounded = round(f_1_step_pred_price/100, 0) * 100
        f_1_step_pred_dynamic_increment = global_parm_dynamic_increment
        f_1_step_pred_set_price_rounded = f_1_step_pred_price_rounded + f_1_step_pred_dynamic_increment
        f_current_step_pred_les = f_1_step_pred_les
        f_current_step_pred_les_misc = f_1_step_pred_adj_misc
        f_current_step_pred_price_inc = f_1_step_pred_price_inc
        f_current_step_pred_price = f_1_step_pred_price
        f_current_step_pred_price_rounded = f_1_step_pred_price_rounded
        f_current_step_pred_dynamic_increment = f_1_step_pred_dynamic_increment # +200 or + 300
        f_current_step_pred_set_price_rounded = f_1_step_pred_set_price_rounded
        f_current_step_error = f_current_step_pred_price_inc - f_actual_price4pm # current second forecast error
    else:
        previous_time = shl_intra_fetch_previous_n_sec_time_as_str(shl_data_history_ts_process['time'][i], 1)
        previous_pred_les_level = shl_data_shl_pm[(shl_data_shl_pm['ccyy-mm'] == shl_data_history_ts_process['ccyy-mm'][i]) \
                                            & (shl_data_shl_pm['time'] ==previous_time)].iloc[0]['f_1_step_pred_les_level']
        print('     previous_pred_les_level : %f' % previous_pred_les_level)
        
        previous_pred_les_trend = shl_data_shl_pm[(shl_data_shl_pm['ccyy-mm'] == shl_data_history_ts_process['ccyy-mm'][i]) \
                                            & (shl_data_shl_pm['time'] ==previous_time)].iloc[0]['f_1_step_pred_les_trend']
        print('     previous_pred_les_trend : %f' % previous_pred_les_trend)
        
        f_current_step_pred_les = shl_data_shl_pm[(shl_data_shl_pm['ccyy-mm'] == shl_data_history_ts_process['ccyy-mm'][i]) \
                                                    & (shl_data_shl_pm['time'] ==previous_time)].iloc[0]['f_1_step_pred_les']
        f_current_step_pred_les_misc = shl_data_shl_pm[(shl_data_shl_pm['ccyy-mm'] == shl_data_history_ts_process['ccyy-mm'][i]) \
                                                    & (shl_data_shl_pm['time'] ==previous_time)].iloc[0]['f_1_step_pred_adj_misc']
        f_current_step_pred_price_inc = shl_data_shl_pm[(shl_data_shl_pm['ccyy-mm'] == shl_data_history_ts_process['ccyy-mm'][i]) \
                                                    & (shl_data_shl_pm['time'] ==previous_time)].iloc[0]['f_1_step_pred_price_inc']
        f_current_step_pred_price = shl_data_shl_pm[(shl_data_shl_pm['ccyy-mm'] == shl_data_history_ts_process['ccyy-mm'][i]) \
                                                    & (shl_data_shl_pm['time'] ==previous_time)].iloc[0]['f_1_step_pred_price']
        f_current_step_pred_price_rounded = shl_data_shl_pm[(shl_data_shl_pm['ccyy-mm'] == shl_data_history_ts_process['ccyy-mm'][i]) \
                                                    & (shl_data_shl_pm['time'] ==previous_time)].iloc[0]['f_1_step_pred_price_rounded']
        f_current_step_pred_dynamic_increment = shl_data_shl_pm[(shl_data_shl_pm['ccyy-mm'] == shl_data_history_ts_process['ccyy-mm'][i]) \
                                                    & (shl_data_shl_pm['time'] ==previous_time)].iloc[0]['f_1_step_pred_dynamic_increment']
        f_current_step_pred_set_price_rounded = shl_data_shl_pm[(shl_data_shl_pm['ccyy-mm'] == shl_data_history_ts_process['ccyy-mm'][i]) \
                                                    & (shl_data_shl_pm['time'] ==previous_time)].iloc[0]['f_1_step_pred_set_price_rounded']

        f_current_step_error = f_current_step_pred_price_inc - f_actual_price4pm # current second forecast error
            
        if shl_data_history_ts_process['time'][i] == '11:29:50':
            # function to get average forecast error between 46~50 seconds: mean(f_current_step_error)
            global_parm_short_weight_misc = (shl_data_shl_pm.iloc[46:50]['f_current_step_error'].sum() \
                                             + f_current_step_error) / 5
            print('*INFO* global_parm_short_weight_misc : %f' % global_parm_short_weight_misc)
            
#         call prediction functino shl_pm, forcaste next k=1 step
        print('---- call prediction function shl_pm ---- %s' % shl_data_history_ts_process['time'][i])
        
        f_1_step_pred_les_level = global_parm_alpha * f_actual_price4pmsi \
                                    + (1 - global_parm_alpha) * (previous_pred_les_level + previous_pred_les_trend)
        print('     f_1_step_pred_les_level  : %f' % f_1_step_pred_les_level)
        f_1_step_pred_les_trend = global_parm_beta * (f_1_step_pred_les_level - previous_pred_les_level) \
                                    + (1 - global_parm_beta) * previous_pred_les_trend
        print('     f_1_step_pred_les_trend  : %f' % f_1_step_pred_les_trend)
        f_1_step_pred_les = f_1_step_pred_les_level + f_1_step_pred_les_trend
        
        f_1_step_pred_adj_misc = global_parm_short_weight_misc * global_parm_short_weight * global_parm_gamma
        
#         f_1_step_pred_price_inc = (f_1_step_pred_les + f_1_step_pred_adj_misc) * f_actual_si
        f_1_step_pred_price_inc = (f_1_step_pred_les + f_1_step_pred_adj_misc) * f_1_step_si
        f_1_step_pred_price = f_1_step_pred_price_inc + global_parm_base_price
        f_1_step_pred_price_rounded = round(f_1_step_pred_price/100, 0) * 100
        f_1_step_pred_dynamic_increment = global_parm_dynamic_increment
        f_1_step_pred_set_price_rounded = f_1_step_pred_price_rounded + f_1_step_pred_dynamic_increment
   
        
    # write results to shl_pm dataframe
            
    shl_data_shl_pm_current = {
                         'ccyy-mm' : shl_data_history_ts_process['ccyy-mm'][i]
                        ,'time' : shl_data_history_ts_process['time'][i]
                        ,'bid' : shl_data_history_ts_process['bid-price'][i]
                        ,'datetime' : f_actual_datetime
                        ,'f_actual_price4pm' : f_actual_price4pm
                        ,'f_actual_si' : f_actual_si
                        ,'f_1_step_si' : f_1_step_si
                        ,'f_actual_price4pmsi' :  f_actual_price4pmsi
                        ,'f_1_step_pred_les_level' : f_1_step_pred_les_level
                        ,'f_1_step_pred_les_trend' : f_1_step_pred_les_trend
                        ,'f_1_step_pred_les' : f_1_step_pred_les
                        ,'f_1_step_pred_adj_misc' : f_1_step_pred_adj_misc
                        ,'f_1_step_pred_price_inc' : f_1_step_pred_price_inc
                        ,'f_1_step_pred_price' : f_1_step_pred_price
                        ,'f_1_step_pred_price_rounded' : f_1_step_pred_price_rounded
                        ,'f_1_step_pred_dynamic_increment' : f_1_step_pred_dynamic_increment # +200 or + 300
                        ,'f_1_step_pred_set_price_rounded' : f_1_step_pred_set_price_rounded
                        ,'f_current_step_pred_les' : f_current_step_pred_les
                        ,'f_current_step_pred_les_misc' : f_current_step_pred_les_misc
                        ,'f_current_step_pred_price_inc' : f_current_step_pred_price_inc
                        ,'f_current_step_pred_price' : f_current_step_pred_price
                        ,'f_current_step_pred_price_rounded' : f_current_step_pred_price_rounded
                        ,'f_current_step_pred_dynamic_increment' : f_current_step_pred_dynamic_increment # +200 or + 300
                        ,'f_current_step_pred_set_price_rounded' : f_current_step_pred_set_price_rounded
                        ,'f_current_step_error' : f_current_step_error
                        }
    shl_data_shl_pm =  shl_data_shl_pm.append(shl_data_shl_pm_current, ignore_index=True)

In [ ]:
# shl_data_shl_pm.iloc[2]
shl_data_shl_pm.head()

In [ ]:
shl_data_shl_pm.tail()

In [ ]:
plt.figure(figsize=(12,6))
plt.plot(shl_data_shl_pm['bid'])
plt.plot(shl_data_shl_pm['f_current_step_pred_price'])
# plt.plot(shl_data_shl_pm['f_1_step_pred_price'].shift(1))

plt.figure(figsize=(12,6))
plt.plot(shl_data_shl_pm['bid'])
plt.plot(shl_data_shl_pm['f_current_step_pred_price'])
plt.plot(shl_data_shl_pm['f_1_step_pred_price'])

In [ ]:

Start of prediction module: shl_pm


In [ ]:
# 11:29:51~
def predict_k_step_price(shl_data_shl_pm, ccyy_mm, time, k):
    print('month & time  : ', ccyy_mm, time)
    print()
    
#     shl_data_shl_pm_k = pd.DataFrame() # initialize prediction dataframe
    for sec in range(0, k):
        
        
        print('delta second(s) : ', sec)
        current_time  = shl_intra_fetch_future_n_sec_time_as_str(time, sec)
        print('current_time  : %s' % current_time)
        f_1_step_time  = shl_intra_fetch_future_n_sec_time_as_str(current_time, 1)
        print('f_1_step_time  : %s' % f_1_step_time)
        previous_time = shl_intra_fetch_previous_n_sec_time_as_str(current_time, 1)
        print('previous_time : %s' % previous_time)

        previous_pred_les_level = shl_data_shl_pm[(shl_data_shl_pm['ccyy-mm'] == global_parm_ccyy_mm) \
                                            & (shl_data_shl_pm['time'] ==previous_time)].iloc[0]['f_1_step_pred_les_level']
        print('     previous_pred_les_level : %f' % previous_pred_les_level)
        
        previous_pred_les_trend = shl_data_shl_pm[(shl_data_shl_pm['ccyy-mm'] == global_parm_ccyy_mm) \
                                            & (shl_data_shl_pm['time'] ==previous_time)].iloc[0]['f_1_step_pred_les_trend']
        print('     previous_pred_les_trend : %f' % previous_pred_les_trend)


        print('---- Pre-Process ---')
        ############ use predicted value for boost-trap
        previous_pred_price = shl_data_shl_pm[(shl_data_shl_pm['ccyy-mm'] == global_parm_ccyy_mm) \
                                            & (shl_data_shl_pm['time'] == previous_time)].iloc[0]['f_1_step_pred_price']
        # pre-process: ccyy-mm-hh:mm:ss
        f_actual_datetime = global_parm_ccyy_mm + ' ' + current_time
#         f_actual_price4pm = shl_data_history_ts_process['bid-price'][i] -  global_parm_base_price
        f_actual_price4pm = previous_pred_price -  global_parm_base_price
        print('*INFO* f_actual_datetime   : %s ' %  f_actual_datetime)
        print('*INFO* previous_pred_price: %s ' %  previous_pred_price)
        print('*INFO* f_actual_price4pm   : %d ' % f_actual_price4pm)

        # get Seasonality-Index
        f_actual_si = shl_intra_fetch_si(global_parm_ccyy_mm
                                             ,current_time
                                             ,shl_data_parm_si)
        try:
            f_1_step_si = shl_intra_fetch_si(global_parm_ccyy_mm
                                                 ,f_1_step_time
                                                 ,shl_data_parm_si)
        except:
            f_1_step_si = shl_intra_fetch_si(global_parm_ccyy_mm
                                                 ,current_time
                                                 ,shl_data_parm_si)            

        print('*INFO* f_actual_si         : %0.10f ' %  f_actual_si)
        # get de-seasoned price: price4pmsi
        f_actual_price4pmsi = f_actual_price4pm / f_actual_si
        print('*INFO* f_actual_price4pmsi : %0.10f ' % f_actual_price4pmsi)

        f_current_step_pred_les = shl_data_shl_pm[(shl_data_shl_pm['ccyy-mm'] == shl_data_history_ts_process['ccyy-mm'][i]) \
                                                    & (shl_data_shl_pm['time'] ==previous_time)].iloc[0]['f_1_step_pred_les']
        f_current_step_pred_les_misc = shl_data_shl_pm[(shl_data_shl_pm['ccyy-mm'] == shl_data_history_ts_process['ccyy-mm'][i]) \
                                                    & (shl_data_shl_pm['time'] ==previous_time)].iloc[0]['f_1_step_pred_adj_misc']
        f_current_step_pred_price_inc = shl_data_shl_pm[(shl_data_shl_pm['ccyy-mm'] == shl_data_history_ts_process['ccyy-mm'][i]) \
                                                    & (shl_data_shl_pm['time'] ==previous_time)].iloc[0]['f_1_step_pred_price_inc']
        f_current_step_pred_price = shl_data_shl_pm[(shl_data_shl_pm['ccyy-mm'] == shl_data_history_ts_process['ccyy-mm'][i]) \
                                                    & (shl_data_shl_pm['time'] ==previous_time)].iloc[0]['f_1_step_pred_price']
        f_current_step_pred_price_rounded = shl_data_shl_pm[(shl_data_shl_pm['ccyy-mm'] == shl_data_history_ts_process['ccyy-mm'][i]) \
                                                    & (shl_data_shl_pm['time'] ==previous_time)].iloc[0]['f_1_step_pred_price_rounded']
        f_current_step_pred_dynamic_increment = shl_data_shl_pm[(shl_data_shl_pm['ccyy-mm'] == shl_data_history_ts_process['ccyy-mm'][i]) \
                                                    & (shl_data_shl_pm['time'] ==previous_time)].iloc[0]['f_1_step_pred_dynamic_increment']
        f_current_step_pred_set_price_rounded = shl_data_shl_pm[(shl_data_shl_pm['ccyy-mm'] == shl_data_history_ts_process['ccyy-mm'][i]) \
                                                    & (shl_data_shl_pm['time'] ==previous_time)].iloc[0]['f_1_step_pred_set_price_rounded']
        
        f_current_step_error = f_current_step_pred_price_inc - f_actual_price4pm # current second forecast error
        

        f_1_step_pred_les_level = global_parm_alpha * f_actual_price4pmsi \
                                    + (1 - global_parm_alpha) * (previous_pred_les_level + previous_pred_les_trend)
        print('     f_1_step_pred_les_level  : %f' % f_1_step_pred_les_level)
        f_1_step_pred_les_trend = global_parm_beta * (f_1_step_pred_les_level - previous_pred_les_level) \
                                    + (1 - global_parm_beta) * previous_pred_les_trend
        print('     f_1_step_pred_les_trend  : %f' % f_1_step_pred_les_trend)
        f_1_step_pred_les = f_1_step_pred_les_level + f_1_step_pred_les_trend
        
#         f_1_step_pred_adj_misc = 0
        f_1_step_pred_adj_misc = global_parm_short_weight_misc * global_parm_short_weight * (sec+2) * global_parm_gamma
        
#         f_1_step_pred_price_inc = (f_1_step_pred_les + f_1_step_pred_adj_misc) * f_actual_si
        f_1_step_pred_price_inc = (f_1_step_pred_les + f_1_step_pred_adj_misc) * f_1_step_si
        f_1_step_pred_price = f_1_step_pred_price_inc + global_parm_base_price
        f_1_step_pred_price_rounded = round(f_1_step_pred_price/100, 0) * 100
        f_1_step_pred_dynamic_increment = global_parm_dynamic_increment
        f_1_step_pred_set_price_rounded = f_1_step_pred_price_rounded + f_1_step_pred_dynamic_increment 

#         write results to shl_pm dataframe
        shl_data_shl_pm_current = {
                             'ccyy-mm' : global_parm_ccyy_mm
                            ,'time' : current_time
                            ,'bid' : previous_pred_price
                            ,'datetime' : f_actual_datetime
                            ,'f_actual_price4pm' : f_actual_price4pm
                            ,'f_actual_si' : f_actual_si
                            ,'f_1_step_si' : f_1_step_si
                            ,'f_actual_price4pmsi' :  f_actual_price4pmsi
                            ,'f_1_step_pred_les_level' : f_1_step_pred_les_level
                            ,'f_1_step_pred_les_trend' : f_1_step_pred_les_trend
                            ,'f_1_step_pred_les' : f_1_step_pred_les
                            ,'f_1_step_pred_adj_misc' : f_1_step_pred_adj_misc
                            ,'f_1_step_pred_price_inc' : f_1_step_pred_price_inc
                            ,'f_1_step_pred_price' : f_1_step_pred_price
                            ,'f_1_step_pred_price_rounded' : f_1_step_pred_price_rounded
                            ,'f_1_step_pred_dynamic_increment' : f_1_step_pred_dynamic_increment # +200 or + 300
                            ,'f_1_step_pred_set_price_rounded' : f_1_step_pred_set_price_rounded
                            ,'f_current_step_pred_les' : f_current_step_pred_les
                            ,'f_current_step_pred_les_misc' : f_current_step_pred_les_misc
                            ,'f_current_step_pred_price_inc' : f_current_step_pred_price_inc
                            ,'f_current_step_pred_price' : f_current_step_pred_price
                            ,'f_current_step_pred_price_rounded' : f_current_step_pred_price_rounded
                            ,'f_current_step_pred_dynamic_increment' : f_current_step_pred_dynamic_increment # +200 or + 300
                            ,'f_current_step_pred_set_price_rounded' : f_current_step_pred_set_price_rounded
                            ,'f_current_step_error' : f_current_step_error
                            }
        print('---------------------------')
        shl_data_shl_pm =  shl_data_shl_pm.append(shl_data_shl_pm_current, ignore_index=True)
        
    return shl_data_shl_pm

In [ ]:
shl_data_shl_pm_k_step = predict_k_step_price(shl_data_shl_pm, global_parm_ccyy_mm, '11:29:51', 10)

In [ ]:
shl_data_shl_pm_k_step['f_current_step_pred_les_misc'].tail(11)

In [ ]:
# bid is predicted bid-price from shl_pm
plt.figure(figsize=(12,6))
plt.plot(shl_data_shl_pm_k_step['bid'])
# plt.plot(shl_data_shl_pm_k_step['f_1_step_pred_price'].shift(1))
plt.plot(shl_data_shl_pm_k_step['f_current_step_pred_price'])

# bid is actual bid-price from raw dataset
shl_data_actual_bid = shl_data_history_ts_process[global_parm_ccyy_mm_offset:global_parm_ccyy_mm_offset+61].copy()
shl_data_actual_bid.reset_index(inplace=True)
plt.figure(figsize=(12,6))
plt.plot(shl_data_actual_bid['bid-price'])
# plt.plot(shl_data_shl_pm_k_step['f_1_step_pred_price'].shift(1))
plt.plot(shl_data_shl_pm_k_step['f_current_step_pred_price'])
# plt.plot(shl_data_shl_pm_k_step['bid'])

End of prediction module: shl_pm


In [ ]:


In [ ]:
# actual price & oredicted price & error
pd.concat([shl_data_actual_bid['bid-price'].tail(11), shl_data_shl_pm_k_step['f_current_step_pred_price'].tail(11), shl_data_shl_pm_k_step['f_current_step_pred_price'].tail(11) - shl_data_actual_bid['bid-price'].tail(11)], axis=1, join='inner')

In [ ]:
shl_data_shl_pm_k_step.iloc[57]
# shl_data_shl_pm_k_step.iloc[50:61]

In [ ]:

End of shl_sm


In [ ]:


In [ ]:


In [ ]:


In [ ]:


In [ ]:


In [ ]:


In [ ]:


In [ ]:


In [ ]:


In [ ]:


In [ ]:


In [ ]:


In [ ]:


In [ ]:


The End